{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "couldn't import doomish\n",
      "Couldn't import doom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:187: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: The result returned by `env.reset()` was not a tuple of the form `(obs, info)`, where `obs` is a observation and `info` is a dictionary containing additional information. Actual type: `<class 'numpy.ndarray'>`\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([24.       ,  0.       , 24.       , 32.       , 24.       ,\n",
       "       36.       ,  7.1824338])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "import gym_pygame\n",
    "\n",
    "#!pip install git+https://github.com/ntasfi/PyGame-Learning-Environment.git\n",
    "#!pip install git+https://github.com/qlan3/gym-games.git\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pong-PLE-v0')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box(-inf, inf, (7,), float32)\n",
      "env.action_space= Discrete(3)\n",
      "state= [ 24.           0.          24.          32.          24.\n",
      "  36.         -10.07757721]\n",
      "action= 0\n",
      "next_state= [ 23.424       -0.576       23.04        33.2         23.66408076\n",
      "  36.         -10.07757721]\n",
      "reward= 0.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:219: DeprecationWarning: \u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to rewrite the environment with new step API. \u001b[0m\n",
      "  logger.deprecation(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method is not within the observation space.\u001b[0m\n",
      "  logger.warn(f\"{pre} is not within the observation space.\")\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:272: UserWarning: \u001b[33mWARN: No render modes was declared in the environment (env.metadata['render_modes'] is None or not defined), you may have trouble when calling `.render()`.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARcAAADdCAYAAACR1zayAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAR4UlEQVR4nO3dbWxT5fsH8O9G1zPG1o4N17JshSUuToIoDBgVExLXSJQoD4vhhUZUItkouMEbshdgTNQu8kLFIPgQwURkZiagI1FcBsyQlMFKpjw5IJCscWsnMT0dD3vIev3f/P4nFthGt962G99PciXuPvc55+Ke/ebsnG5NEREBEVGcpSa6ASKanBguRKQEw4WIlGC4EJESDBciUoLhQkRKMFyISAmGCxEpwXAhIiUYLkSkhLJw2b17N2bPno309HSUlZXh9OnTqk5FREkoRcXvFn3//fd47bXXsHfvXpSVleHjjz9GQ0MDOjo6kJeXN+K+kUgEXV1dyMrKQkpKSrxbI6JxEhH09vYiPz8fqakjXJ+IAosXLxa32218PTQ0JPn5+eLxeEbd1+/3CwAWi5Xk5ff7R3wtx/3HooGBAfh8PrhcLmMsNTUVLpcLXq/3nvn9/f0Ih8NGCX9Jm2hCyMrKGnF73MPlxo0bGBoags1mixq32WwIBAL3zPd4PLBarUY5HI54t0RECox22yLhT4tqa2uh67pRfr8/0S0RURyY4n3AGTNmYMqUKQgGg1HjwWAQdrv9nvmapkHTtHi3QUQJFvcrF7PZjNLSUjQ3NxtjkUgEzc3NcDqd8T4dESWr8T4Zup/6+nrRNE32798vFy9elA0bNkh2drYEAoFR99V1PeF3wVks1uil6/qIr+W4/1gEAGvXrsXff/+NHTt2IBAI4KmnnsIvv/xyz01eIpq8lLyJbjzC4TCsVmui2yCiUei6DovFMuz2hD8tIqLJieFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKMFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSIuZw+e233/Diiy8iPz8fKSkpOHz4cNR2EcGOHTswc+ZMTJ06FS6XC1euXIlXv0Q0QcQcLrdu3cKTTz6J3bt333f7hx9+iF27dmHv3r1obW3FtGnTsHz5cvT19Y27WSKaQEb8mPpRAJBDhw4ZX0ciEbHb7bJz505jLBQKiaZpcvDgwQc6pq7rAoDFYiV56bo+4ms5rvdcrl+/jkAgAJfLZYxZrVaUlZXB6/Xed5/+/n6Ew+GoIqKJL67hEggEAAA2my1q3GazGdvu5vF4YLVajSosLIxnS0SUIAl/WlRbWwtd143y+/2JbomI4iCu4WK32wEAwWAwajwYDBrb7qZpGiwWS1QR0cQX13ApKiqC3W5Hc3OzMRYOh9Ha2gqn0xnPUxFRkjPFusPNmzdx9epV4+vr16+jvb0dOTk5cDgcqKmpwXvvvYfi4mIUFRVh+/btyM/Px6pVq+LZNxElu1gfPx8/fvy+j6XWrVtnPI7evn272Gw20TRNysvLpaOj44GPz0fRLNbEqNEeRaeIiCCJhMNhWK3WRLdBRKPQdX3Ee6QJf1pERJMTw4WIlGC4EJESDBciUoLhQkRKMFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSguFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlIgpXDweDxYtWoSsrCzk5eVh1apV6OjoiJrT19cHt9uN3NxcZGZmoqKiAsFgMK5NE1HyiylcWlpa4Ha7cerUKTQ1NWFwcBDPPfccbt26ZczZsmULGhsb0dDQgJaWFnR1dWHNmjVxb3w4ImIUEQ3v368VJa+XET+mfhQ9PT0CQFpaWkREJBQKSVpamjQ0NBhzLl26JADE6/Xe9xh9fX2i67pRfr9fAIy5/m08x2GxJnvdLdb9dV0fMR/Gdc9F13UAQE5ODgDA5/NhcHAQLpfLmFNSUgKHwwGv13vfY3g8HlitVqMKCwvH0xIRJYkxh0skEkFNTQ2WLl2KuXPnAgACgQDMZjOys7Oj5tpsNgQCgfsep7a2FrquG+X3+8faEhElEdNYd3S73Th//jxOnjw5rgY0TYOmaeM6BhElnzFduWzatAlHjhzB8ePHUVBQYIzb7XYMDAwgFApFzQ8Gg7Db7eNqlIgmmFhu4EYiEXG73ZKfny+XL1++Z/v/39D94YcfjLE///xTgOFv6N5N1/W43aQaz3FYrMled4t1/9Fu6MYULlVVVWK1WuXEiRPS3d1t1O3bt405lZWV4nA45NixY9LW1iZOp1OcTucDn4PhMvlqNInu72Gt8X4f4houw51k3759xpw7d+7Ixo0bZfr06ZKRkSGrV6+W7u7uBz4Hw2Xy1Vj/v2L9t9+XWPcfLVxS/nfQpBEOh2G1Wse8/7//OSkpKfFoicZptP/F+H1KjLu/L7F+H3Rdh8ViGXY7f7eIiJRguBCREgwXIlKC4UJESoz5HbpED4o3bB9OvHIhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpMSke/s/32pO9GBUv1Z45UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSguFCRErEFC579uzBvHnzYLFYYLFY4HQ68fPPPxvb+/r64Ha7kZubi8zMTFRUVCAYDMa9aSJKfjGFS0FBAerq6uDz+dDW1oZnn30WK1euxIULFwAAW7ZsQWNjIxoaGtDS0oKuri6sWbNGSeNElOQe+BPihzF9+nT56quvJBQKSVpamjQ0NBjbLl26JADE6/U+8PHG+0H0LBbrv6nRPoh+zPdchoaGUF9fj1u3bsHpdMLn82FwcBAul8uYU1JSAofDAa/XO+xx+vv7EQ6Ho4qIJr6Yw+XcuXPIzMyEpmmorKzEoUOHMGfOHAQCAZjNZmRnZ0fNt9lsCAQCwx7P4/HAarUaVVhYGPM/goiST8zh8thjj6G9vR2tra2oqqrCunXrcPHixTE3UFtbC13XjfL7/WM+FhElj5j/novZbMajjz4KACgtLcWZM2fwySefYO3atRgYGEAoFIq6egkGg7Db7cMeT9M0aJoWe+dElNTG/T6XSCSC/v5+lJaWIi0tDc3Nzca2jo4OdHZ2wul0jvc0RDTBxHTlUltbi+effx4OhwO9vb347rvvcOLECRw9ehRWqxXr16/H1q1bkZOTA4vFgs2bN8PpdGLJkiWq+ieiZBXLY+c333xTZs2aJWazWR555BEpLy+XX3/91dh+584d2bhxo0yfPl0yMjJk9erV0t3dHcsp+CiaxZogNdqj6BQRESSRcDgMq9Wa6DaIaBS6rsNisQy7nb9bRERKMFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSguFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKjCtc6urqkJKSgpqaGmOsr68Pbrcbubm5yMzMREVFBYLB4Hj7JKIJZszhcubMGXz++eeYN29e1PiWLVvQ2NiIhoYGtLS0oKurC2vWrBl3o0Q0wcT0KfH/09vbK8XFxdLU1CTLli2T6upqEREJhUKSlpYmDQ0NxtxLly4JAPF6vfwgehZrEtVoH0Q/pisXt9uNFStWwOVyRY37fD4MDg5GjZeUlMDhcMDr9d73WP39/QiHw1FFRBOfKdYd6uvrcfbsWZw5c+aebYFAAGazGdnZ2VHjNpsNgUDgvsfzeDx49913Y22DiJJcTFcufr8f1dXVOHDgANLT0+PSQG1tLXRdN8rv98fluESUWDGFi8/nQ09PDxYsWACTyQSTyYSWlhbs2rULJpMJNpsNAwMDCIVCUfsFg0HY7fb7HlPTNFgslqgiookvph+LysvLce7cuaixN954AyUlJdi2bRsKCwuRlpaG5uZmVFRUAAA6OjrQ2dkJp9MZv66JKOnFFC5ZWVmYO3du1Ni0adOQm5trjK9fvx5bt25FTk4OLBYLNm/eDKfTiSVLlsSvayJKejHf0B3NRx99hNTUVFRUVKC/vx/Lly/HZ599Fu/TEFGSSxERSXQT/xYOh2G1WhPdBhGNQtf1Ee+R8neLiEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSguFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKMFyISAmGCxEpwXAhIiWSLlyS7E/6EtEwRnutJl249Pb2JroFInoAo71Wk+6v/0ciEXR1dUFE4HA44Pf7+SmMwwiHwygsLOQaDYPrM7qxrJGIoLe3F/n5+UhNHf76JO6fWzReqampKCgoQDgcBgB+xOsD4BqNjOszuljX6EE+/ifpfiwiosmB4UJESiRtuGiahnfeeQeapiW6laTFNRoZ12d0Ktco6W7oEtHkkLRXLkQ0sTFciEgJhgsRKcFwISIlGC5EpETShsvu3bsxe/ZspKeno6ysDKdPn050Swnh8XiwaNEiZGVlIS8vD6tWrUJHR0fUnL6+PrjdbuTm5iIzMxMVFRUIBoMJ6jix6urqkJKSgpqaGmOM6wP89ddfePXVV5Gbm4upU6fiiSeeQFtbm7FdRLBjxw7MnDkTU6dOhcvlwpUrV8Z3UklC9fX1Yjab5euvv5YLFy7IW2+9JdnZ2RIMBhPd2n9u+fLlsm/fPjl//ry0t7fLCy+8IA6HQ27evGnMqayslMLCQmlubpa2tjZZsmSJPP300wnsOjFOnz4ts2fPlnnz5kl1dbUx/rCvzz///COzZs2S119/XVpbW+XatWty9OhRuXr1qjGnrq5OrFarHD58WH7//Xd56aWXpKioSO7cuTPm8yZluCxevFjcbrfx9dDQkOTn54vH40lgV8mhp6dHAEhLS4uIiIRCIUlLS5OGhgZjzqVLlwSAeL3eRLX5n+vt7ZXi4mJpamqSZcuWGeHC9RHZtm2bPPPMM8Nuj0QiYrfbZefOncZYKBQSTdPk4MGDYz5v0v1YNDAwAJ/PB5fLZYylpqbC5XLB6/UmsLPkoOs6ACAnJwcA4PP5MDg4GLVeJSUlcDgcD9V6ud1urFixImodAK4PAPz0009YuHAhXn75ZeTl5WH+/Pn48ssvje3Xr19HIBCIWiOr1YqysrJxrVHShcuNGzcwNDQEm80WNW6z2RAIBBLUVXKIRCKoqanB0qVLMXfuXABAIBCA2WxGdnZ21NyHab3q6+tx9uxZeDyee7ZxfYBr165hz549KC4uxtGjR1FVVYW3334b33zzDQAY6xDv11zS/ckFGp7b7cb58+dx8uTJRLeSNPx+P6qrq9HU1IT09PREt5OUIpEIFi5ciA8++AAAMH/+fJw/fx579+7FunXrlJ036a5cZsyYgSlTptxzNz8YDMJutyeoq8TbtGkTjhw5guPHj6OgoMAYt9vtGBgYQCgUipr/sKyXz+dDT08PFixYAJPJBJPJhJaWFuzatQsmkwk2m+2hXh8AmDlzJubMmRM19vjjj6OzsxMAjHWI92su6cLFbDajtLQUzc3NxlgkEkFzczOcTmcCO0sMEcGmTZtw6NAhHDt2DEVFRVHbS0tLkZaWFrVeHR0d6OzsfCjWq7y8HOfOnUN7e7tRCxcuxCuvvGL898O8PgCwdOnSe96+cPnyZcyaNQsAUFRUBLvdHrVG4XAYra2t41ujMd8KVqi+vl40TZP9+/fLxYsXZcOGDZKdnS2BQCDRrf3nqqqqxGq1yokTJ6S7u9uo27dvG3MqKyvF4XDIsWPHpK2tTZxOpzidzgR2nVj/flokwvU5ffq0mEwmef/99+XKlSty4MABycjIkG+//daYU1dXJ9nZ2fLjjz/KH3/8IStXrpycj6JFRD799FNxOBxiNptl8eLFcurUqUS3lBAA7lv79u0z5ty5c0c2btwo06dPl4yMDFm9erV0d3cnrukEuztcuD4ijY2NMnfuXNE0TUpKSuSLL76I2h6JRGT79u1is9lE0zQpLy+Xjo6OcZ2Tf8+FiJRIunsuRDQ5MFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKfF/j5QCXksZ6qcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render(mode='rgb_array'))\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x7f1d35fde280>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = PPO(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=1024,  #运行N步后执行更新,buffer_size=n_steps*环境数量\n",
    "    batch_size=64,  #采样数据量\n",
    "    n_epochs=16,  #每次采样后训练的次数\n",
    "    gamma=0.99,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-9.6, 4.454211490264018)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "705b0b5ec0624cefbfcc2ae69f55e965",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:219: DeprecationWarning: \n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">WARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to </span>\n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">rewrite the environment with new step API. </span>\n",
       "  logger.deprecation(\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:219: DeprecationWarning: \n",
       "\u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to \u001b[0m\n",
       "\u001b[33mrewrite the environment with new step API. \u001b[0m\n",
       "  logger.deprecation(\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:141: UserWarning: <span style=\"color: #808000; text-decoration-color: #808000\">WARN: The </span>\n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">obs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64</span>\n",
       "  logger.warn(\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The \u001b[0m\n",
       "\u001b[33mobs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
       "  logger.warn(\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:165: UserWarning: <span style=\"color: #808000; text-decoration-color: #808000\">WARN: The </span>\n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">obs returned by the `step()` method is not within the observation space.</span>\n",
       "  logger.warn(f\"{pre} is not within the observation space.\")\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The \u001b[0m\n",
       "\u001b[33mobs returned by the `step()` method is not within the observation space.\u001b[0m\n",
       "  logger.warn(f\"{pre} is not within the observation space.\")\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:187: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: The result returned by `env.reset()` was not a tuple of the form `(obs, info)`, where `obs` is a observation and `info` is a dictionary containing additional information. Actual type: `<class 'numpy.ndarray'>`\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/3.PPO.Pong')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.1, 0.30000000000000004)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = PPO.load('save/3.PPO.Pong')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARcAAADdCAYAAACR1zayAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAARwElEQVR4nO3dbWxT5fsH8O9G1zNga8eGtCyssETiJBOFAaNiQuIayU+iPCyGFxpRiWRQkMEbshdgTNQu8kLFIPgQwURlpiagkCiSATMkZWMlKIM5IJDQuLVITE/Hwx6yXv83/k8s7IFuvenZ+H6SK2H3uXvOxZ30m7Nz2p0MEREQEaVYZrobIKKxieFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKKAuXXbt2YcaMGcjOzkZ5eTmamppUHYqITChDxXeLvv/+e7z66qvYs2cPysvL8dFHH8Hv96OtrQ1TpkwZ9LXxeBzt7e3Izc1FRkZGqlsjohESEXR2dqKwsBCZmYOcn4gCCxYsEK/Xa/zc19cnhYWF4vP5hnxtKBQSACwWy+QVCoUGfS+n/Neinp4eBINBeDweYywzMxMejweBQOCe+d3d3YjFYkYJv6RNNCrk5uYOuj3l4XLjxg309fXB4XAkjDscDoTD4Xvm+3w+2O12o1wuV6pbIiIFhrpskfa7RTU1NdB13ahQKJTulogoBSyp3uHkyZMxbtw4RCKRhPFIJAKn03nPfE3ToGlaqtsgojRL+ZmL1WpFWVkZ6uvrjbF4PI76+nq43e5UH46IzGqkd4b6U1dXJ5qmyb59++TChQuydu1aycvLk3A4PORrdV1P+1VwFos1dOm6Puh7OeW/FgHAqlWr8Pfff2P79u0Ih8N46qmn8Msvv9xzkZeIxi4lH6IbiVgsBrvdnu42iGgIuq7DZrMNuD3td4uIaGxiuBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSguFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKMFyISAmGCxEpwXAhIiUYLkSkBMOFiJRIOlx+++03vPDCCygsLERGRgYOHjyYsF1EsH37dkydOhXjx4+Hx+PBpUuXUtUvEY0SSYfLrVu38OSTT2LXrl39bv/ggw+wc+dO7NmzB42NjZg4cSKWLFmCrq6uETdLRKPIoI+pHwIAOXDggPFzPB4Xp9MpO3bsMMai0ahomib79++/r33qui4AWCyWyUvX9UHfyym95nL16lWEw2F4PB5jzG63o7y8HIFAoN/XdHd3IxaLJRQRjX4pDZdwOAwAcDgcCeMOh8PYdjefzwe73W5UUVFRKlsiojRJ+92impoa6LpuVCgUSndLRJQCKQ0Xp9MJAIhEIgnjkUjE2HY3TdNgs9kSiohGv5SGS3FxMZxOJ+rr642xWCyGxsZGuN3uVB6KiEzOkuwLbt68icuXLxs/X716FWfPnkV+fj5cLheqq6vx7rvvYubMmSguLsa2bdtQWFiI5cuXp7JvIjK7ZG8/Hz9+vN/bUqtXrzZuR2/btk0cDodomiYVFRXS1tZ23/vnrWgWa3TUULeiM0REYCKxWAx2uz3dbRDREHRdH/QaadrvFhHR2MRwISIlGC5EpATDhYiUYLgQkRIMFyJSguFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKMFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlkgoXn8+H+fPnIzc3F1OmTMHy5cvR1taWMKerqwterxcFBQXIyclBZWUlIpFISpsmIvNLKlwaGhrg9Xpx6tQpHD16FL29vXjuuedw69YtY87mzZtx6NAh+P1+NDQ0oL29HStXrkx540RkcoM+pn4I169fFwDS0NAgIiLRaFSysrLE7/cbc1pbWwWABAKBfvfR1dUluq4bFQqFBACLxTJ56bo+aD6M6JqLrusAgPz8fABAMBhEb28vPB6PMaekpAQulwuBQKDfffh8PtjtdqOKiopG0hIRmcSwwyUej6O6uhqLFi1CaWkpACAcDsNqtSIvLy9hrsPhQDgc7nc/NTU10HXdqFAoNNyWiMhELMN9odfrRUtLC06ePDmiBjRNg6ZpI9oHEZnPsM5cNmzYgMOHD+P48eOYNm2aMe50OtHT04NoNJowPxKJwOl0jqhRIhplkrmAG4/Hxev1SmFhoVy8ePGe7f9/QfeHH34wxv78808BBr6gezdd19N+oYrFYg1dQ13QTSpc1q1bJ3a7XU6cOCEdHR1G3b5925hTVVUlLpdLjh07Js3NzeJ2u8Xtdt/3MRguLNboqJSGy0AH2bt3rzHnzp07sn79epk0aZJMmDBBVqxYIR0dHQwXFmuM1VDhkvFvaJhGLBaD3W5PdxtENARd12Gz2Qbczu8WEZESDBciUoLhQkRKMFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSguFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESSYXL7t27MXv2bNhsNthsNrjdbvz888/G9q6uLni9XhQUFCAnJweVlZWIRCIpb5qIzC+pcJk2bRpqa2sRDAbR3NyMZ599FsuWLcP58+cBAJs3b8ahQ4fg9/vR0NCA9vZ2rFy5UknjRGRyyTyIvj+TJk2SL7/8UqLRqGRlZYnf7ze2tba2CgAJBAL3vT8+iJ7FGh011IPoh33Npa+vD3V1dbh16xbcbjeCwSB6e3vh8XiMOSUlJXC5XAgEAgPup7u7G7FYLKGIaPRLOlzOnTuHnJwcaJqGqqoqHDhwALNmzUI4HIbVakVeXl7CfIfDgXA4POD+fD4f7Ha7UUVFRUn/J4jIfJIOl8ceewxnz55FY2Mj1q1bh9WrV+PChQvDbqCmpga6rhsVCoWGvS8iMg9Lsi+wWq149NFHAQBlZWU4ffo0Pv74Y6xatQo9PT2IRqMJZy+RSAROp3PA/WmaBk3Tku+ciExtxJ9zicfj6O7uRllZGbKyslBfX29sa2trw7Vr1+B2u0d6GCIaZZI6c6mpqcH//vc/uFwudHZ24rvvvsOJEydw5MgR2O12rFmzBlu2bEF+fj5sNhs2btwIt9uNhQsXquqfiMwqmdvOb7zxhkyfPl2sVqs88sgjUlFRIb/++qux/c6dO7J+/XqZNGmSTJgwQVasWCEdHR3JHIK3olmsUVJD3YrOEBGBicRiMdjt9nS3QURD0HUdNpttwO38bhERKcFwISIlGC5EpATDhYiUYLgQkRJJf0KXSKWhbl5mZGQ8oE7GvrvXOtVryzMXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpARvRZOp8Fbz2MEzFyJSguFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKjLmP///3r2vxo+RE6cMzFyJSYkThUltbi4yMDFRXVxtjXV1d8Hq9KCgoQE5ODiorKxGJREbaJxGNMsMOl9OnT+Ozzz7D7NmzE8Y3b96MQ4cOwe/3o6GhAe3t7Vi5cuWIGyWiUSapp8T/q7OzU2bOnClHjx6VxYsXy6ZNm0REJBqNSlZWlvj9fmNua2urAJBAIPBAHkT/XyPZD4s11utuyb5+qAfRD+vMxev1YunSpfB4PAnjwWAQvb29CeMlJSVwuVwIBAL97qu7uxuxWCyhiGj0S/puUV1dHc6cOYPTp0/fsy0cDsNqtSIvLy9h3OFwIBwO97s/n8+Hd955J9k2iMjkkjpzCYVC2LRpE7799ltkZ2enpIGamhroum5UKBRKyX6JaHAZGRkJlWpJhUswGMT169cxd+5cWCwWWCwWNDQ0YOfOnbBYLHA4HOjp6UE0Gk14XSQSgdPp7HefmqbBZrMlFBGNfkn9WlRRUYFz584ljL3++usoKSnB1q1bUVRUhKysLNTX16OyshIA0NbWhmvXrsHtdqeuayIyvaTCJTc3F6WlpQljEydOREFBgTG+Zs0abNmyBfn5+bDZbNi4cSPcbjcWLlyYuq6JyPRS/vH/Dz/8EJmZmaisrER3dzeWLFmCTz/9NNWHISKTy/j3/rZpxGIx2O32Yb9e+N0iogdC1/VBr5Hyu0VEpATDhYiUYLgQkRIMFyJSguFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKjLknLvKb0ETmwDMXIlKC4UJESjBciEgJhgsRKcFwISIlTBcuJvuTvkQ0gKHeq6YLl87OznS3QET3Yaj3qun++n88Hkd7eztEBC6XC6FQiE9hHEAsFkNRURHXaABcn6ENZ41EBJ2dnSgsLERm5sDnJ6b7EF1mZiamTZuGWCwGAHzE633gGg2O6zO0ZNfofh7/Y7pfi4hobGC4EJESpg0XTdPw9ttvQ9O0dLdiWlyjwXF9hqZyjUx3QZeIxgbTnrkQ0ejGcCEiJRguRKQEw4WIlGC4EJESpg2XXbt2YcaMGcjOzkZ5eTmamprS3VJa+Hw+zJ8/H7m5uZgyZQqWL1+Otra2hDldXV3wer0oKChATk4OKisrEYlE0tRxetXW1iIjIwPV1dXGGNcH+Ouvv/DKK6+goKAA48ePxxNPPIHm5mZju4hg+/btmDp1KsaPHw+Px4NLly6N7KBiQnV1dWK1WuWrr76S8+fPy5tvvil5eXkSiUTS3doDt2TJEtm7d6+0tLTI2bNn5fnnnxeXyyU3b9405lRVVUlRUZHU19dLc3OzLFy4UJ5++uk0dp0eTU1NMmPGDJk9e7Zs2rTJGH/Y1+eff/6R6dOny2uvvSaNjY1y5coVOXLkiFy+fNmYU1tbK3a7XQ4ePCi///67vPjii1JcXCx37twZ9nFNGS4LFiwQr9dr/NzX1yeFhYXi8/nS2JU5XL9+XQBIQ0ODiIhEo1HJysoSv99vzGltbRUAEggE0tXmA9fZ2SkzZ86Uo0ePyuLFi41w4fqIbN26VZ555pkBt8fjcXE6nbJjxw5jLBqNiqZpsn///mEf13S/FvX09CAYDMLj8RhjmZmZ8Hg8CAQCaezMHHRdBwDk5+cDAILBIHp7exPWq6SkBC6X66FaL6/Xi6VLlyasA8D1AYCffvoJ8+bNw0svvYQpU6Zgzpw5+OKLL4ztV69eRTgcTlgju92O8vLyEa2R6cLlxo0b6Ovrg8PhSBh3OBwIh8Np6soc4vE4qqursWjRIpSWlgIAwuEwrFYr8vLyEuY+TOtVV1eHM2fOwOfz3bON6wNcuXIFu3fvxsyZM3HkyBGsW7cOb731Fr7++msAMNYh1e850/3JBRqY1+tFS0sLTp48me5WTCMUCmHTpk04evQosrOz092OKcXjccybNw/vv/8+AGDOnDloaWnBnj17sHr1amXHNd2Zy+TJkzFu3Lh7ruZHIhE4nc40dZV+GzZswOHDh3H8+HFMmzbNGHc6nejp6UE0Gk2Y/7CsVzAYxPXr1zF37lxYLBZYLBY0NDRg586dsFgscDgcD/X6AMDUqVMxa9ashLHHH38c165dAwBjHVL9njNduFitVpSVlaG+vt4Yi8fjqK+vh9vtTmNn6SEi2LBhAw4cOIBjx46huLg4YXtZWRmysrIS1qutrQ3Xrl17KNaroqIC586dw9mzZ42aN28eXn75ZePfD/P6AMCiRYvu+fjCxYsXMX36dABAcXExnE5nwhrFYjE0NjaObI2GfSlYobq6OtE0Tfbt2ycXLlyQtWvXSl5enoTD4XS39sCtW7dO7Ha7nDhxQjo6Ooy6ffu2MaeqqkpcLpccO3ZMmpubxe12i9vtTmPX6fXfu0UiXJ+mpiaxWCzy3nvvyaVLl+Tbb7+VCRMmyDfffGPMqa2tlby8PPnxxx/ljz/+kGXLlo3NW9EiIp988om4XC6xWq2yYMECOXXqVLpbSgsA/dbevXuNOXfu3JH169fLpEmTZMKECbJixQrp6OhIX9Npdne4cH1EDh06JKWlpaJpmpSUlMjnn3+esD0ej8u2bdvE4XCIpmlSUVEhbW1tIzom/54LESlhumsuRDQ2MFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKfF/L7WB4LnLYgoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 400 [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 2 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
